﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data.SqlClient;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

namespace SynonymNormalizer
{
    class Program
    {
        private static string _connectionString = ConfigurationManager.ConnectionStrings["Freebase"].ConnectionString;

        public static void Main(string[] args)
        {
            using(var connection = new SqlConnection(_connectionString))
            using(var cmd = connection.CreateCommand())
            {
                cmd.CommandText =
@"select MID, Destination
from dbo.Dump
where property in ('/type/object/key', '/common/topic/alias', '/type/object/name')
	and (
		value in (
		'/en',
        '/lang/en'
		)
		or 
		value like '/wikipedia/%' and value not like '%[_]id'
	)
    and Destination is not null
";
                connection.Open();

                int counter = 0;

                using (var reader = cmd.ExecuteReader())
                using (StreamWriter file = new StreamWriter(@"C:\Users\Егор\Documents\synonyms_normalized.txt", false, Encoding.Unicode))
                {
                    
                    while (reader.Read())
                    {
                        string mid = reader.GetString(0);
                        string synonym = reader.GetString(1);

                        file.WriteLine(string.Join("\t", mid, Normalize(synonym)));

                        int count = Interlocked.Increment(ref counter);
                        if (count % 1000000 == 0)
                            Console.WriteLine(count + " synos done");
                    }
                }
            }

        }

        public static string Normalize(string syno)
        {
            var enc = new UnicodeEncoding();
            var regex = new Regex(@"\$(?<digits>[\dABCDEF]{4})");
            Match match;
            while ((match = regex.Match(syno)).Value != String.Empty)
            {
                string digits = match.Groups["digits"].Value;
                if (String.IsNullOrEmpty(digits)) continue;
                int numberChars = digits.Length;
                var bytes = new byte[numberChars / 2];
                for (int i = 0; i < digits.Length; i += 2)
                {
                    bytes[i / 2] = Convert.ToByte(digits.Substring(i, 2), 16);
                }
                byte x = bytes[0];
                bytes[0] = bytes[1];
                bytes[1] = x;
                string a = enc.GetString(bytes);
                if (a.Length > 0
                    && char.GetUnicodeCategory(a[0]) != UnicodeCategory.OtherLetter
                    && char.GetUnicodeCategory(a[0]) != UnicodeCategory.Control
                    && char.GetUnicodeCategory(a[0]) != UnicodeCategory.Format)
                { syno = syno.Replace(match.Value, a); }
                else
                    return null;
            }
            syno = syno.Replace("_", " ");
            syno = syno.Replace("&amp;", "&");
            return syno;
        }

    }
}
